require File.expand_path('../../config/environment',  __FILE__)

require 'thread'

#Rails.logger = Logger.new $stdout

task_queue = Queue.new
empty_count = 0
last_id = 0

threads = []
agent_aliases = Mechanize::AGENT_ALIASES.keys
agent_aliases.delete 'Mechanize'
10.times.each do
  threads << Thread.new do
    while true
      if Thread.current[:agent].nil?
        agent = Mechanize.new
        agent.user_agent_alias = agent_aliases[rand agent_aliases.size]
        Thread.current[:agent] = agent
      else
        Thread.current[:agent].agent.history.clear #清理历史，以释放到page的引用
      end
      begin
        if task_queue.empty?
          sleep 3
          next
        end
        list_task = task_queue.pop
        list_task.create_for_cat
      rescue ThreadError => e
        Rails.logger.info "线程任务完成 #{Thread.current.object_id} , #{e.message}"
        break
      rescue Exception => e
        Rails.logger.warn "采集分类出错, #{list_task.id}, #{e.message}, #{e.backtrace.join '\n'}"
      end
    end
  end
end

while true
  if task_queue.size > 5
    sleep 3
    next
  end

  list_tasks = ListTask.all(:conditions => "id > #{last_id}", :order => 'id ASC', :limit => 20)
  if list_tasks.empty?
    empty_count += 1
    exit if empty_count > 5
    sleep 20*empty_count
    next
  else
    empty_count = 0
    last_id = list_tasks.last.id
    list_tasks.each{|list_task| task_queue << list_task}
    sleep 3
  end
end

